

<!DOCTYPE html>
<html lang="zh-CN" data-default-color-scheme=auto>



<head><meta name="generator" content="Hexo 3.9.0">
  <meta charset="UTF-8">
  <link rel="apple-touch-icon" sizes="76x76" href="/img/favicon.ico">
  <link rel="icon" href="/img/favicon.ico">
  <meta name="viewport" content="width=device-width, initial-scale=1.0, maximum-scale=5.0, shrink-to-fit=no">
  <meta http-equiv="x-ua-compatible" content="ie=edge">
  
  <meta name="theme-color" content="#2f4154">
  <meta name="description" content="Try not to become a man of success but rather try to become a man of value. ">
  <meta name="author" content="kinzun">
  <meta name="keywords" content>
  <meta name="description" content="重点为介绍 asyncio。 asyncio  可以实现单线程并发 IO 操作。Asynchronous HTTP Client/Server for asyncio and Python。 aiohttp 则是基于 asyncio 实现的 HTTP 框架。  AIOHTTP  我们先安装 aiohttp： 1pip install aiohttp 编写一个 Flask 服务器，模拟网络爬虫场景">
<meta name="keywords" content="Python">
<meta property="og:type" content="article">
<meta property="og:title" content="线程，协程对比和 Python 爬虫使用">
<meta property="og:url" content="https://kinzun.github.io/2019/04/10/python-协程-异步-爬虫对比/index.html">
<meta property="og:site_name" content="kinzun">
<meta property="og:description" content="重点为介绍 asyncio。 asyncio  可以实现单线程并发 IO 操作。Asynchronous HTTP Client/Server for asyncio and Python。 aiohttp 则是基于 asyncio 实现的 HTTP 框架。  AIOHTTP  我们先安装 aiohttp： 1pip install aiohttp 编写一个 Flask 服务器，模拟网络爬虫场景">
<meta property="og:locale" content="zh-CN">
<meta property="og:updated_time" content="2019-05-15T12:49:02.211Z">
<meta name="twitter:card" content="summary_large_image">
<meta name="twitter:title" content="线程，协程对比和 Python 爬虫使用">
<meta name="twitter:description" content="重点为介绍 asyncio。 asyncio  可以实现单线程并发 IO 操作。Asynchronous HTTP Client/Server for asyncio and Python。 aiohttp 则是基于 asyncio 实现的 HTTP 框架。  AIOHTTP  我们先安装 aiohttp： 1pip install aiohttp 编写一个 Flask 服务器，模拟网络爬虫场景">
  
  <title>线程，协程对比和 Python 爬虫使用 - kinzun</title>

  <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/bootstrap@4/dist/css/bootstrap.min.css">


  <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/github-markdown-css@4/github-markdown.min.css">
  <link rel="stylesheet" href="/lib/hint/hint.min.css">

  
    
    
      
      <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/highlight.js@10/styles/github-gist.min.css">
    
  

  
    <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/@fancyapps/fancybox@3/dist/jquery.fancybox.min.css">
  


<!-- 主题依赖的图标库，不要自行修改 -->
<link rel="stylesheet" href="//at.alicdn.com/t/font_1749284_ba1fz6golrf.css">

<link rel="stylesheet" href="//at.alicdn.com/t/font_1736178_kmeydafke9r.css">

<link rel="stylesheet" href="/css/main.css">

<!-- 自定义样式保持在最底部 -->


  <script id="fluid-configs">
    var Fluid = window.Fluid || {};
    var CONFIG = {"hostname":"kinzun.github.io","root":"/","version":"1.8.12","typing":{"enable":true,"typeSpeed":70,"cursorChar":"_","loop":false},"anchorjs":{"enable":true,"element":"h1,h2,h3,h4,h5,h6","placement":"right","visible":"hover","icon":""},"progressbar":{"enable":true,"height_px":3,"color":"#29d","options":{"showSpinner":false,"trickleSpeed":100}},"copy_btn":true,"image_zoom":{"enable":true,"img_url_replace":["",""]},"toc":{"enable":true,"headingSelector":"h1,h2,h3,h4,h5,h6","collapseDepth":0},"lazyload":{"enable":true,"loading_img":"/img/loading.gif","onlypost":false,"offset_factor":2},"web_analytics":{"enable":false,"baidu":null,"google":null,"gtag":null,"tencent":{"sid":null,"cid":null},"woyaola":null,"cnzz":null,"leancloud":{"app_id":null,"app_key":null,"server_url":null,"path":"window.location.pathname"}},"search_path":"/local-search.xml"};
  </script>
  <script src="/js/utils.js"></script>
  <script src="/js/color-schema.js"></script>
</head>


<body>
  <header style="height: 70vh;">
    <nav id="navbar" class="navbar fixed-top  navbar-expand-lg navbar-dark scrolling-navbar">
  <div class="container">
    <a class="navbar-brand" href="/">
      <strong>kinzun</strong>
    </a>

    <button id="navbar-toggler-btn" class="navbar-toggler" type="button" data-toggle="collapse"
            data-target="#navbarSupportedContent"
            aria-controls="navbarSupportedContent" aria-expanded="false" aria-label="Toggle navigation">
      <div class="animated-icon"><span></span><span></span><span></span></div>
    </button>

    <!-- Collapsible content -->
    <div class="collapse navbar-collapse" id="navbarSupportedContent">
      <ul class="navbar-nav ml-auto text-center">
        
          
          
          
          
            <li class="nav-item">
              <a class="nav-link" href="/">
                <i class="iconfont icon-home-fill"></i>
                首页
              </a>
            </li>
          
        
          
          
          
          
            <li class="nav-item">
              <a class="nav-link" href="/archives/">
                <i class="iconfont icon-archive-fill"></i>
                归档
              </a>
            </li>
          
        
          
          
          
          
            <li class="nav-item">
              <a class="nav-link" href="/categories/">
                <i class="iconfont icon-category-fill"></i>
                分类
              </a>
            </li>
          
        
          
          
          
          
            <li class="nav-item">
              <a class="nav-link" href="/tags/">
                <i class="iconfont icon-tags-fill"></i>
                标签
              </a>
            </li>
          
        
          
          
          
          
            <li class="nav-item">
              <a class="nav-link" href="/about/">
                <i class="iconfont icon-user-fill"></i>
                关于
              </a>
            </li>
          
        
        
          <li class="nav-item" id="search-btn">
            <a class="nav-link" target="_self" href="javascript:;" data-toggle="modal" data-target="#modalSearch" aria-label="Search">
              &nbsp;<i class="iconfont icon-search"></i>&nbsp;
            </a>
          </li>
        
        
          <li class="nav-item" id="color-toggle-btn">
            <a class="nav-link" target="_self" href="javascript:;" aria-label="Color Toggle">&nbsp;<i
                class="iconfont icon-dark" id="color-toggle-icon"></i>&nbsp;</a>
          </li>
        
      </ul>
    </div>
  </div>
</nav>

    <div class="banner" id="banner" parallax=true
         style="background: url('/img/default.png') no-repeat center center;
           background-size: cover;">
      <div class="full-bg-img">
        <div class="mask flex-center" style="background-color: rgba(0, 0, 0, 0.3)">
          <div class="page-header text-center fade-in-up">
            <span class="h2" id="subtitle" title="线程，协程对比和 Python 爬虫使用">
              
            </span>

            
              <div class="mt-3">
  
  
    <span class="post-meta">
      <i class="iconfont icon-date-fill" aria-hidden="true"></i>
      <time datetime="2019-04-10 10:38" pubdate>
        2019年4月10日 上午
      </time>
    </span>
  
</div>

<div class="mt-1">
  
    <span class="post-meta mr-2">
      <i class="iconfont icon-chart"></i>
      4.6k 字
    </span>
  

  
    <span class="post-meta mr-2">
      <i class="iconfont icon-clock-fill"></i>
      
      
      14 分钟
    </span>
  

  
  
</div>

            
          </div>

          
        </div>
      </div>
    </div>
  </header>

  <main>
    
      

<div class="container-fluid nopadding-x">
  <div class="row nomargin-x">
    <div class="d-none d-lg-block col-lg-2"></div>
    <div class="col-lg-8 nopadding-x-md">
      <div class="container nopadding-x-md" id="board-ctn">
        <div class="py-5" id="board">
          <article class="post-content mx-auto">
            <!-- SEO header -->
            <h1 style="display: none">线程，协程对比和 Python 爬虫使用</h1>
            
            <div class="markdown-body">
              <p>重点为介绍 <code>asyncio</code>。</p>
<p><code>asyncio</code>  可以实现单线程并发 IO 操作。Asynchronous HTTP Client/Server for <a href="https://aiohttp.readthedocs.io/en/stable/glossary.html#term-asyncio" target="_blank" rel="noopener">asyncio</a> and Python。</p>
<p>aiohttp <code>则是基于</code> asyncio 实现的 HTTP 框架。</p>
<ul>
<li><a href="https://aiohttp.readthedocs.io/en/stable/" target="_blank" rel="noopener">AIOHTTP</a></li>
</ul>
<p>我们先安装 <code>aiohttp</code>：</p>
<figure class="highlight shell"><table><tr><td class="gutter hljs"><div class="hljs code-wrapper"><pre><span class="line">1</span><br></pre></div></td><td class="code"><div class="hljs code-wrapper"><pre><code class="hljs shell">pip install aiohttp<br></code></pre></div></td></tr></table></figure>
<p>编写一个 Flask 服务器，模拟网络爬虫场景</p>
<figure class="highlight python"><table><tr><td class="gutter hljs"><div class="hljs code-wrapper"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br></pre></div></td><td class="code"><div class="hljs code-wrapper"><pre><code class="hljs python"><span class="hljs-keyword">from</span> flask <span class="hljs-keyword">import</span> Flask<br><span class="hljs-keyword">import</span> time<br><br>app = Flask(__name__)<br><span class="hljs-meta">@app.route('/')</span><br><span class="hljs-function"><span class="hljs-keyword">def</span> <span class="hljs-title">index</span><span class="hljs-params">()</span>:</span><br>    time.sleep(<span class="hljs-number">3</span>)<br>    <span class="hljs-keyword">return</span> <span class="hljs-string">'Hello ,world!'</span><br><br><br><br><span class="hljs-meta">@app.route('/go')</span><br><span class="hljs-function"><span class="hljs-keyword">def</span> <span class="hljs-title">go</span><span class="hljs-params">()</span>:</span><br>    time.sleep(<span class="hljs-number">3</span>)<br>    <span class="hljs-keyword">return</span> <span class="hljs-string">'Hello ,go!'</span><br><br><br><span class="hljs-meta">@app.route('/python')</span><br><span class="hljs-function"><span class="hljs-keyword">def</span> <span class="hljs-title">python</span><span class="hljs-params">()</span>:</span><br>    time.sleep(<span class="hljs-number">3</span>)<br>    <span class="hljs-keyword">return</span> <span class="hljs-string">'Hello ,python!'</span><br><br><br><span class="hljs-meta">@app.route('/c')</span><br><span class="hljs-function"><span class="hljs-keyword">def</span> <span class="hljs-title">c</span><span class="hljs-params">()</span>:</span><br>    time.sleep(<span class="hljs-number">3</span>)<br>    <span class="hljs-keyword">return</span> <span class="hljs-string">'Hello ,c!'</span><br><br><br><span class="hljs-keyword">if</span> __name__ == <span class="hljs-string">'__main__'</span>:<br>    app.run(threaded=<span class="hljs-keyword">True</span>, port=<span class="hljs-number">5000</span>)<br></code></pre></div></td></tr></table></figure>
<h4 id="绑定回调"><a href="#绑定回调" class="headerlink" title="绑定回调"></a>绑定回调</h4><figure class="highlight python"><table><tr><td class="gutter hljs"><div class="hljs code-wrapper"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br></pre></div></td><td class="code"><div class="hljs code-wrapper"><pre><code class="hljs python"><span class="hljs-keyword">import</span> asyncio<br><span class="hljs-keyword">import</span> requests<br> <br><span class="hljs-keyword">async</span> <span class="hljs-function"><span class="hljs-keyword">def</span> <span class="hljs-title">request</span><span class="hljs-params">()</span>:</span><br>    url = <span class="hljs-string">'https://www.baidu.com'</span><br>    status = requests.get(url)<br>    <span class="hljs-keyword">return</span> status<br> <br><span class="hljs-function"><span class="hljs-keyword">def</span> <span class="hljs-title">callback</span><span class="hljs-params">(task)</span>:</span><br>    print(<span class="hljs-string">'Status:'</span>, task.result())<br> <br>coroutine = request()<br>task = asyncio.ensure_future(coroutine)<br>task.add_done_callback(callback)<br>print(<span class="hljs-string">'Task:'</span>, task)<br> <br>loop = asyncio.get_event_loop()<br>loop.run_until_complete(task)<br>print(<span class="hljs-string">'Task:'</span>, task)<br></code></pre></div></td></tr></table></figure>
<p>多线程协程爬虫，测试是并行</p>
<figure class="highlight python"><table><tr><td class="gutter hljs"><div class="hljs code-wrapper"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br><span class="line">32</span><br><span class="line">33</span><br><span class="line">34</span><br><span class="line">35</span><br><span class="line">36</span><br><span class="line">37</span><br><span class="line">38</span><br><span class="line">39</span><br><span class="line">40</span><br></pre></div></td><td class="code"><div class="hljs code-wrapper"><pre><code class="hljs python"><span class="hljs-keyword">import</span> time<br><span class="hljs-keyword">import</span> requests<br><span class="hljs-keyword">import</span> asyncio<br><span class="hljs-keyword">from</span> multiprocessing.dummy <span class="hljs-keyword">import</span> Pool <span class="hljs-keyword">as</span> ThreadPool<br><br>path = [<span class="hljs-string">"/go"</span>, <span class="hljs-string">"/c"</span>, <span class="hljs-string">"/python"</span>, <span class="hljs-string">"/"</span>] * <span class="hljs-number">100</span><br>host = <span class="hljs-string">"http://127.0.0.1:5000"</span><br>full_url = list(map(<span class="hljs-keyword">lambda</span> x: <span class="hljs-string">f"<span class="hljs-subst">&#123;host&#125;</span><span class="hljs-subst">&#123;x&#125;</span>"</span>, path))<br><br>thread = <span class="hljs-number">4</span><br><br><br><span class="hljs-keyword">async</span> <span class="hljs-function"><span class="hljs-keyword">def</span> <span class="hljs-title">get_page</span><span class="hljs-params">(url, loop)</span>:</span><br>    future = loop.run_in_executor(<br>        <span class="hljs-keyword">None</span>, requests.get, url<br>    )<br>    response = <span class="hljs-keyword">await</span> future<br>    print(response.text)<br><br><br><span class="hljs-function"><span class="hljs-keyword">def</span> <span class="hljs-title">divide</span><span class="hljs-params">(i)</span>:</span><br>    <span class="hljs-keyword">import</span> asyncio<br>    loop = asyncio.new_event_loop()<br>    asyncio.set_event_loop(loop)<br><br>    tasks = [asyncio.ensure_future(get_page(url, loop))<br>             <span class="hljs-keyword">for</span> url <span class="hljs-keyword">in</span> full_url]<br>    loop.run_until_complete(asyncio.wait(tasks))<br>    loop.close()<br><br><br><span class="hljs-keyword">if</span> __name__ == <span class="hljs-string">'__main__'</span>:<br>    start = time.time()<br>    pool = ThreadPool(thread)<br>    <span class="hljs-comment"># i = [j for j in range(0, thread)]</span><br>    pool.map(divide, )<br>    pool.close()<br>    pool.join()<br>    print(<span class="hljs-string">"爬取&#123;0&#125;个网页 ，总花费时间:&#123;1:.2f&#125;s"</span>.format(<br>        len(path), start - time.time()), end=<span class="hljs-string">""</span>)<br></code></pre></div></td></tr></table></figure>
<figure class="highlight python"><table><tr><td class="gutter hljs"><div class="hljs code-wrapper"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br></pre></div></td><td class="code"><div class="hljs code-wrapper"><pre><code class="hljs python"><span class="hljs-keyword">import</span> asyncio<br><span class="hljs-keyword">import</span> aiohttp<br><span class="hljs-keyword">import</span> time<br><span class="hljs-keyword">from</span> aiomultiprocess <span class="hljs-keyword">import</span> Pool<br> <br>start = time.time()<br> <br><span class="hljs-keyword">async</span> <span class="hljs-function"><span class="hljs-keyword">def</span> <span class="hljs-title">get</span><span class="hljs-params">(url)</span>:</span><br>    session = aiohttp.ClientSession()<br>    response = <span class="hljs-keyword">await</span> session.get(url)<br>    result = <span class="hljs-keyword">await</span> response.text()<br>    session.close()<br>    <span class="hljs-keyword">return</span> result<br> <br><span class="hljs-keyword">async</span> <span class="hljs-function"><span class="hljs-keyword">def</span> <span class="hljs-title">request</span><span class="hljs-params">()</span>:</span><br>    url = <span class="hljs-string">'http://127.0.0.1:5000'</span><br>    urls = [url <span class="hljs-keyword">for</span> _ <span class="hljs-keyword">in</span> range(<span class="hljs-number">100</span>)]<br>    <span class="hljs-keyword">async</span> <span class="hljs-keyword">with</span> Pool() <span class="hljs-keyword">as</span> pool:<br>        result = <span class="hljs-keyword">await</span> pool.map(get, urls)<br>        <span class="hljs-keyword">return</span> result<br> <br>coroutine = request()<br>task = asyncio.ensure_future(coroutine)<br>loop = asyncio.get_event_loop()<br>loop.run_until_complete(task)<br> <br>end = time.time()<br>print(<span class="hljs-string">'Cost time:'</span>, end - start)<br></code></pre></div></td></tr></table></figure>
<h4 id="异步协程协程-request-代码"><a href="#异步协程协程-request-代码" class="headerlink" title="异步协程协程 request 代码"></a>异步协程协程 request 代码</h4><figure class="highlight python"><table><tr><td class="gutter hljs"><div class="hljs code-wrapper"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br></pre></div></td><td class="code"><div class="hljs code-wrapper"><pre><code class="hljs python"><span class="hljs-keyword">import</span> time<br><span class="hljs-keyword">import</span> requests<br><span class="hljs-keyword">import</span> asyncio<br><br>path = [<span class="hljs-string">"/go"</span>, <span class="hljs-string">"/c"</span>, <span class="hljs-string">"/python"</span>, <span class="hljs-string">"/"</span>] * <span class="hljs-number">100</span><br>host = <span class="hljs-string">"http://127.0.0.1:5000"</span><br>full_url = list(map(<span class="hljs-keyword">lambda</span> x: <span class="hljs-string">f"<span class="hljs-subst">&#123;host&#125;</span><span class="hljs-subst">&#123;x&#125;</span>"</span>, path))<br><br><br><span class="hljs-keyword">async</span> <span class="hljs-function"><span class="hljs-keyword">def</span> <span class="hljs-title">get_page</span><span class="hljs-params">(url, loop)</span>:</span><br>    future = loop.run_in_executor(<br>        <span class="hljs-keyword">None</span>, requests.get, url<br>    )<br>    response = <span class="hljs-keyword">await</span> future<br>    <span class="hljs-comment"># print(response.text)</span><br><br><br><span class="hljs-keyword">if</span> __name__ == <span class="hljs-string">'__main__'</span>:<br>    start = time.time()<br>    loop = asyncio.get_event_loop()<br>    tasks = [asyncio.ensure_future(get_page(url, loop)) <span class="hljs-keyword">for</span> url <span class="hljs-keyword">in</span> full_url]<br>    loop.run_until_complete(asyncio.wait(tasks))<br>    loop.close()<br>    print(<span class="hljs-string">"爬取&#123;0&#125;个网页 ，总花费时间:&#123;1:.2f&#125;s"</span>.format(<br>        len(path), start - time.time()), end=<span class="hljs-string">""</span>)<br></code></pre></div></td></tr></table></figure>
<p>输出结果: 爬取400个网页 ，总花费时间:-21.20s</p>
<h4 id="异步协程爬虫"><a href="#异步协程爬虫" class="headerlink" title="异步协程爬虫"></a>异步协程爬虫</h4><p>因 requests 不支持异步。换 <code>asyncio</code></p>
<p>代码代码如下：</p>
<figure class="highlight python"><table><tr><td class="gutter hljs"><div class="hljs code-wrapper"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br></pre></div></td><td class="code"><div class="hljs code-wrapper"><pre><code class="hljs python"><span class="hljs-keyword">import</span> time<br><span class="hljs-keyword">import</span> aiohttp<br><span class="hljs-keyword">import</span> asyncio<br><br><br><span class="hljs-keyword">async</span> <span class="hljs-function"><span class="hljs-keyword">def</span> <span class="hljs-title">get_page</span><span class="hljs-params">(url)</span>:</span><br>    <span class="hljs-keyword">async</span> <span class="hljs-keyword">with</span>  aiohttp.ClientSession() <span class="hljs-keyword">as</span> session:<br>        <span class="hljs-keyword">async</span> <span class="hljs-keyword">with</span> <span class="hljs-keyword">await</span> session.get(url=url) <span class="hljs-keyword">as</span> response:<br>            page_text = <span class="hljs-keyword">await</span> response.text()<br>            print(page_text)<br><br><br>path = [<span class="hljs-string">"/go"</span>, <span class="hljs-string">"/c"</span>, <span class="hljs-string">"/python"</span>, <span class="hljs-string">"/"</span>] * <span class="hljs-number">100</span><br>url = <span class="hljs-string">"http://127.0.0.1:5000"</span><br>full_url = list(map(<span class="hljs-keyword">lambda</span> x: url + x, path))<br><br><span class="hljs-comment">#</span><br><span class="hljs-keyword">if</span> __name__ == <span class="hljs-string">'__main__'</span>:<br>    print(full_url)<br>    start = time.time()<br>    loop = asyncio.get_event_loop()<br>    tasks = [asyncio.ensure_future(get_page(url)) <span class="hljs-keyword">for</span> url <span class="hljs-keyword">in</span> full_url]<br><br>    loop.run_until_complete(asyncio.wait(tasks))<br>    print(<span class="hljs-string">"爬取&#123;0&#125;个网页 ，总花费时间:&#123;1:.2f&#125;s"</span>.format(<br>        len(path), start - time.time()), end=<span class="hljs-string">""</span>)<br></code></pre></div></td></tr></table></figure>
<p>输出结果: 爬取400个网页 ，总花费时间:-3.35s</p>
<h4 id="如何实现数据解析—任务的绑定回调机制"><a href="#如何实现数据解析—任务的绑定回调机制" class="headerlink" title="如何实现数据解析—任务的绑定回调机制"></a>如何实现数据解析—任务的绑定回调机制</h4><figure class="highlight python"><table><tr><td class="gutter hljs"><div class="hljs code-wrapper"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br><span class="line">32</span><br></pre></div></td><td class="code"><div class="hljs code-wrapper"><pre><code class="hljs python"><span class="hljs-comment">### 如何实现数据解析---任务的绑定回调机制</span><br><span class="hljs-keyword">import</span> aiohttp<br><span class="hljs-keyword">import</span> asyncio<br><span class="hljs-comment">#回调函数：解析响应数据</span><br><span class="hljs-function"><span class="hljs-keyword">def</span> <span class="hljs-title">callback</span><span class="hljs-params">(task)</span>:</span><br>    print(<span class="hljs-string">'this is callback()'</span>)<br>    <span class="hljs-comment">#获取响应数据</span><br>    page_text = task.result()<br>    print(<span class="hljs-string">'在回调函数中，实现数据解析'</span>)<br>    <br><span class="hljs-keyword">async</span> <span class="hljs-function"><span class="hljs-keyword">def</span> <span class="hljs-title">get_page</span><span class="hljs-params">(url)</span>:</span><br>    <span class="hljs-keyword">async</span> <span class="hljs-keyword">with</span> aiohttp.ClientSession() <span class="hljs-keyword">as</span> session:<br>        <span class="hljs-keyword">async</span> <span class="hljs-keyword">with</span> <span class="hljs-keyword">await</span> session.get(url=url) <span class="hljs-keyword">as</span> response:<br>            page_text = <span class="hljs-keyword">await</span> response.text() <span class="hljs-comment">#read()  json()</span><br><span class="hljs-comment">#             print(page_text)</span><br>            <span class="hljs-keyword">return</span> page_text<br>start = time.time()<br>path = [<span class="hljs-string">"/go"</span>, <span class="hljs-string">"/c"</span>, <span class="hljs-string">"/python"</span>, <span class="hljs-string">"/"</span>] * <span class="hljs-number">100</span><br>host = <span class="hljs-string">"http://127.0.0.1:5000"</span><br>full_url = list(map(<span class="hljs-keyword">lambda</span> x: <span class="hljs-string">f"<span class="hljs-subst">&#123;host&#125;</span><span class="hljs-subst">&#123;x&#125;</span>"</span>, path))<br><br><br>tasks = []<br>loop = asyncio.get_event_loop()<br><span class="hljs-keyword">for</span> url <span class="hljs-keyword">in</span> urls:<br>    c = get_page(url)<br>    task = asyncio.ensure_future(c)<br>    <span class="hljs-comment">#给任务对象绑定回调函数用于解析响应数据</span><br>    task.add_done_callback(callback)<br>    tasks.append(task)<br>loop.run_until_complete(asyncio.wait(tasks))<br>print(<span class="hljs-string">'总耗时：'</span>,time.time()-start)<br></code></pre></div></td></tr></table></figure>
<p>结合代理池，爬虫速度能做到极致。</p>
<h4 id="参考资料："><a href="#参考资料：" class="headerlink" title="参考资料："></a>参考资料：</h4><ul>
<li><a href="https://github.com/zhang0peter/python-coroutine" target="_blank" rel="noopener">线程，协程对比和Python爬虫实战说明</a></li>
<li><p><a href="https://mp.weixin.qq.com/s?__biz=MjM5MzgyODQxMQ==&amp;mid=2650368555&amp;idx=1&amp;sn=a449f107c9c16466c51ce8a6939fcb1b&amp;chksm=be9cd17f89eb5869c00e964e42e79400d4c9b993c4c5764ddbf9ef0e4b85741fc4ab05c77dbc&amp;mpshare=1&amp;scene=23&amp;srcid=07163jZEvRwfwwii9F8dKopl#rd" target="_blank" rel="noopener">asyncio：高性能异步模块使用介绍</a></p>
</li>
<li><p><a href="https://cuiqingcai.com/6160.html" target="_blank" rel="noopener">Python中异步协程的使用方法介绍</a></p>
</li>
<li><a href="https://www.ibm.com/developerworks/cn/analytics/library/ba-on-demand-data-python-3/index.html" target="_blank" rel="noopener">在 Python 中按需处理数据，第 3 部分 协程和 asyncio</a></li>
<li><a href="https://stackoverflow.com/questions/22190403/how-could-i-use-requests-in-asyncio" target="_blank" rel="noopener">How could I use requests in asyncio?</a></li>
</ul>

            </div>
            <hr>
            <div>
              <div class="post-metas mb-3">
                
                
                  <div class="post-meta">
                    <i class="iconfont icon-tags"></i>
                    
                      <a class="hover-with-bg" href="/tags/Python/">Python</a>
                    
                  </div>
                
              </div>
              
                <p class="note note-warning">
                  
                    本博客所有文章除特别声明外，均采用 <a href="https://creativecommons.org/licenses/by-sa/4.0/deed.zh" rel="nofollow noopener">CC BY-SA 4.0 协议</a> ，转载请注明出处！
                  
                </p>
              
              
                <div class="post-prevnext">
                  <article class="post-prev col-6">
                    
                    
                  </article>
                  <article class="post-next col-6">
                    
                    
                      <a href="/2019/04/01/Python GeventWebSocket/">
                        <span class="hidden-mobile">Python GeventWebSocket</span>
                        <span class="visible-mobile">下一篇</span>
                        <i class="iconfont icon-arrowright"></i>
                      </a>
                    
                  </article>
                </div>
              
            </div>

            
              <!-- Comments -->
              <article class="comments" id="comments" lazyload>
                
                  
                
                

              </article>
            
          </article>
        </div>
      </div>
    </div>
    
      <div class="d-none d-lg-block col-lg-2 toc-container" id="toc-ctn">
        <div id="toc">
  <p class="toc-header"><i class="iconfont icon-list"></i>&nbsp;目录</p>
  <div class="toc-body" id="toc-body"></div>
</div>

      </div>
    
  </div>
</div>

<!-- Custom -->


    

    
      <a id="scroll-top-button" aria-label="TOP" href="#" role="button">
        <i class="iconfont icon-arrowup" aria-hidden="true"></i>
      </a>
    

    
      <div class="modal fade" id="modalSearch" tabindex="-1" role="dialog" aria-labelledby="ModalLabel"
     aria-hidden="true">
  <div class="modal-dialog modal-dialog-scrollable modal-lg" role="document">
    <div class="modal-content">
      <div class="modal-header text-center">
        <h4 class="modal-title w-100 font-weight-bold">搜索</h4>
        <button type="button" id="local-search-close" class="close" data-dismiss="modal" aria-label="Close">
          <span aria-hidden="true">&times;</span>
        </button>
      </div>
      <div class="modal-body mx-3">
        <div class="md-form mb-5">
          <input type="text" id="local-search-input" class="form-control validate">
          <label data-error="x" data-success="v"
                 for="local-search-input">关键词</label>
        </div>
        <div class="list-group" id="local-search-result"></div>
      </div>
    </div>
  </div>
</div>
    

    
  </main>

  <footer class="text-center mt-5 py-3">
  <div class="footer-content">
     <a href="https://hexo.io" target="_blank" rel="nofollow noopener"><span>Hexo</span></a> <i class="iconfont icon-love"></i> <a href="https://github.com/fluid-dev/hexo-theme-fluid" target="_blank" rel="nofollow noopener"><span>Fluid</span></a> 
  </div>
  

  

  
</footer>


  <!-- SCRIPTS -->
  
  <script  src="https://cdn.jsdelivr.net/npm/nprogress@0/nprogress.min.js" ></script>
  <link  rel="stylesheet" href="https://cdn.jsdelivr.net/npm/nprogress@0/nprogress.min.css" />

  <script>
    NProgress.configure({"showSpinner":false,"trickleSpeed":100})
    NProgress.start()
    window.addEventListener('load', function() {
      NProgress.done();
    })
  </script>


<script  src="https://cdn.jsdelivr.net/npm/jquery@3/dist/jquery.min.js" ></script>
<script  src="https://cdn.jsdelivr.net/npm/bootstrap@4/dist/js/bootstrap.min.js" ></script>
<script  src="/js/events.js" ></script>
<script  src="/js/plugins.js" ></script>

<!-- Plugins -->


  <script  src="/js/local-search.js" ></script>



  
    <script  src="/js/img-lazyload.js" ></script>
  



  



  
    <script  src="https://cdn.jsdelivr.net/npm/tocbot@4/dist/tocbot.min.js" ></script>
  
  
    <script  src="https://cdn.jsdelivr.net/npm/@fancyapps/fancybox@3/dist/jquery.fancybox.min.js" ></script>
  
  
    <script  src="https://cdn.jsdelivr.net/npm/anchor-js@4/anchor.min.js" ></script>
  
  
    <script defer src="https://cdn.jsdelivr.net/npm/clipboard@2/dist/clipboard.min.js" ></script>
  






  <script  src="https://cdn.jsdelivr.net/npm/typed.js@2/lib/typed.min.js" ></script>
  <script>
    (function (window, document) {
      var typing = Fluid.plugins.typing;
      var title = document.getElementById('subtitle').title;
      
      typing(title)
      
    })(window, document);
  </script>















<!-- 主题的启动项 保持在最底部 -->
<script  src="/js/boot.js" ></script>


</body>
</html>
